from scrapy import Spider, Request
from scrapy.http import HtmlResponse

from WeatherChecker.items import WeatherItem


class WeatherSpider(Spider):
    name = "Weather"
    allowed_domains = ["tianqi.2345.com"]

    def start_requests(self):
        with open('update_id.txt', 'r') as fp:
            for region_id in fp.readlines():
                region_id = region_id.strip()
                url = f'http://tianqi.2345.com/today-{region_id}.htm'
                yield Request(url)

    def parse(self, response: HtmlResponse, **kwargs):
        # 获取市名
        region = (response.xpath('//*[@class="location-city"]/text()')
                  .extract_first().replace('7天天气详情', ''))
        # 获取所有区的网页url
        region_urls = response.xpath('//*[@class="live-heart" and '
                                     '@style="clear: both;"]/div[1]/ul[1]/li/a/@href').extract()
        for region_url in region_urls:
            # 切出id
            region_id = region_url.split('/')[2].split('.')[0]
            # 补全url, 获取7天的天气
            region_url = f'http://tianqi.2345.com/today-{region_id}.htm'
            yield Request(region_url, callback=self.district_parse, meta={'region': region})

    def district_parse(self, response: HtmlResponse, **kwargs):
        region = response.meta['region']
        # 获取item对象 准备开始组装
        item = WeatherItem()

        # 设置市名称 区名称
        item['region'] = region
        item['district'] = (response.xpath('//*[@class="location-city"]/text()')
                            .extract_first().replace('7天天气详情', ''))
        item['_id'] = item['region'] + '-' + item['district']

        # pm2.5浓度和aqi值只有当天的, 所以就存一次
        # 有的会没有pm2.5浓度数据, 如成都都江堰(https://tianqi.2345.com/today-60407.htm)
        try:
            item['pm_avg'] = response.xpath('//*[@class="aqi-map-style-tip"]/em/text()').extract_first()
        except ValueError:
            item['pm_avg'] = -1
        try:
            item['aqi'] = response.xpath('//*[@class="aqi-map-con"]/span/text()').extract_first()
        except ValueError:
            item['aqi'] = -1

        # 组装7天每天的天气
        weather_info = {}
        info_list = response.xpath('//*[@class="seven-day"]/ul/li[position()>1]')  # 需要去掉昨天的天气信息
        for info in info_list:
            # 组装未来每天天气信息
            one_day_info = {}
            tpr_info = info.xpath('./a/span[3]/text()').extract_first().split('~')
            one_day_info['max_tpr'] = int(tpr_info[1].replace('°', ''))
            one_day_info['min_tpr'] = int(tpr_info[0])
            one_day_info['conditions'] = info.xpath('./a/i/text()').extract_first()
            # 以日期为key添加未来每天天气信息到weather_info
            date = info.xpath('./a/em/text()').extract_first()
            weather_info[date] = one_day_info
        item['weather_info'] = weather_info
        yield item
